Benchmarking DA methods

library(tidyverse)

Warning message:
In system("timedatectl", intern = TRUE) :
  running command 'timedatectl' had status 1

# library(MouseGastrulationData)
library(SingleCellExperiment)
library(scater)
library(scran)
library(bit64)

Loading required package: bit

Attaching package: 'bit'

The following object is masked from 'package:base':

    xor

Attaching package bit64
package:bit64 (c) 2011-2017 Jens Oehlschlaegel
creators: integer64 runif64 seq :
coercion: as.integer64 as.vector as.logical as.integer as.double as.character as.bitstring
logical operator: ! & | xor != == < <= >= >
arithmetic operator: + - * / %/% %% ^
math: sign abs sqrt log log2 log10
math: floor ceiling trunc round
querying: is.integer64 is.vector [is.atomic} [length] format print str
values: is.na is.nan is.finite is.infinite
aggregation: any all min max range sum prod
cumulation: diff cummin cummax cumsum cumprod
access: length<- [ [<- [[ [[<-
combine: c rep cbind rbind as.data.frame
WARNING don't use as subscripts
WARNING semantics differ from integer
for more help type ?bit64

Attaching package: 'bit64'

The following objects are masked from 'package:SummarizedExperiment':

    match, order, rank

The following object is masked from 'package:Biobase':

    cache

The following objects are masked from 'package:GenomicRanges':

    match, order, rank

The following objects are masked from 'package:IRanges':

    match, order

The following objects are masked from 'package:S4Vectors':

    %in%, match, order, rank

The following objects are masked from 'package:BiocGenerics':

    %in%, match, order, rank

The following objects are masked from 'package:base':

    :, %in%, is.double, match, order, rank

library(patchwork)
library(ggpubr)

source('~/milo/notebooks/benchmark/benchmark_utils.R')

figdir <- "~/mount/gdrive/milo/Figures/benchmark_v2/"
if (!dir.exists(figdir)) {
  dir.create(figdir)  
}

cannot create dir '/home/jovyan/mount/gdrive/milo/Figures/benchmark_v2', reason 'No such file or directory'

Some plotting utils

## color palette for methods
method_names <- c("milo", "daseq", "meld", "louvain", "cydar")
method_colors <- RColorBrewer::brewer.pal(7, name="Dark2")[c(1:4,6)]
method_colors <- setNames(method_colors, method_names)
method_labels <- c("Milo", "DAseq", "MELD", "Louvain", "cydar")
method_labels <- setNames(method_labels, method_names)

scale_color_methods <- function(){ scale_color_manual(values = method_colors, labels=method_labels) }
scale_fill_methods <- function(){ scale_fill_manual(values = method_colors, labels=method_labels) }

data.frame(method_names) %>%
  ggplot(aes(method_names, 2, color=method_names )) + geom_point(size=5) +
  scale_color_methods()

enr_labeller2 <- function(value){
  fc <- round(prob2FC(as.numeric(value)), 2)
  enr_name <- paste("logFC = ", fc)
  return(c(value=enr_name))
}

pl_labeller <- labeller(method=as_labeller(setNames(method_labels, method_names)),
     enr=enr_labeller2)

In this notebook I will visualize the results from benchmarking Milo against 4 alternative DA analysis methods.

Benchmarking workflow

All the main functions are implemented in benchmark/benchmark_utils.R. These functions are wrapped in several scripts:

• Condition labels are simulated by running make_bm_data.R (wrapped in submit_make_bm_data.sh)
• Methods implemented in R are run with run_DA_R.r
• MELD is run in run_meld.py
• All of it is wrapped in submit_benchmark.sh (contains parameters for each input dataset)

Define input datasets

These are going to be used as underlying KNN graphs with different topologies for benchmarking.

Simulated datasets

Provided by Mike, constructed with dyntoy

Prepare datasets (uniform naming of dim reduction and celltype column)

branch_sce_3x <- readRDS("~/mount/gdrive/milo/BranchingSimulation-3X.RDS")
branch_sce_3x$cell <- colnames(branch_sce_3x)
branch_sce_3x$celltype <- branch_sce_3x$group_id
names(reducedDims(branch_sce_3x)) <- c("pca.corrected", "UMAP")
saveRDS(branch_sce_3x, "/nfs/team205/ed6/data/milo_benchmark/branching3x_data_bm.RDS")

linear_sce_3x <- readRDS("~/mount/gdrive/milo/LinearTrajSyntheticLabels-3X.RDS")
linear_sce_3x$cell <- colnames(linear_sce_3x)
linear_sce_3x$celltype <- linear_sce_3x$group_id
names(reducedDims(linear_sce_3x)) <- c("pca.corrected", "UMAP")
saveRDS(linear_sce_3x, "/nfs/team205/ed6/data/milo_benchmark/linear3x_data_bm.RDS")

cluster_sce_3x <- readRDS("~/mount/gdrive/milo/DiscreteSyntheticLabels-3X.RDS")
cluster_sce_3x$cell <- colnames(cluster_sce_3x)
cluster_sce_3x$celltype <- cluster_sce_3x$Block
names(reducedDims(cluster_sce_3x)) <- c("pca.corrected", "UMAP")
saveRDS(cluster_sce_3x, "/nfs/team205/ed6/data/milo_benchmark/cluster3x_data_bm.RDS")

Mouse gastrulation data

Select samples from late time points (even number of replicates)

AtlasSampleMetadata %>%
  arrange(stage) %>%
  mutate(sample=factor(sample, levels=unique(sample))) %>%
  ggplot(aes(stage, sample)) +
  geom_point()

late_samples <- AtlasSampleMetadata %>%
  filter(stage %in% c("E7.75", "E8.0", "E8.25", "E8.5")) %>%
  pull(sample)

embryo_sce <- EmbryoAtlasData(type="processed", samples = late_samples)

data.frame(colData(embryo_sce)) %>%
  ggplot(aes(as.factor(sample), as.factor(pool))) + geom_point()

embryo_sce

Preprocessing

logcounts(embryo_sce) <- log1p(counts(embryo_sce))
## Exclude zero counts genes
keep.rows <- rowSums(logcounts(embryo_sce)) != 0
embryo_sce <- embryo_sce[keep.rows, ]

dec <- modelGeneVar(embryo_sce)
hvgs <- getTopHVGs(dec, n=5000)

## Drop cells with NAs in corrected pca (low quality)
embryo_sce <- embryo_sce[,apply(reducedDim(embryo_sce, "pca.corrected"), 1, function(x) all(!is.na(x)))]

## Run UMAP (the data has been subsetted)
# embryo_sce <- scater::runPCA(embryo_sce, subset_row=hvgs)
embryo_sce <- scater::runUMAP(embryo_sce, dimred="pca.corrected")

plotReducedDim(embryo_sce, "UMAP", colour_by="celltype", text_by="celltype", point_size=0.1) +
  scale_color_manual(values=EmbryoCelltypeColours) +
  guides(color=guide_legend(override.aes = list(size=2)))

Save object 4 benchmark

saveRDS(embryo_sce, "/nfs/team205/ed6/data/milo_benchmark/embryo_data_bm.RDS")

embryo_sce <- readRDS("/nfs/team205/ed6/data/milo_benchmark/embryo_data_bm.RDS")

Save list of celltype names and sample by sizes

for (i in 1:3) {
  pop_size_df <- as.data.frame(table(embryo_sce$celltype)) %>%
  rename(pop=Var1, pop_size=Freq) %>%
  mutate(size_bin = cut(pop_size, breaks = 10)) %>%
  group_by(size_bin) %>%
  sample_n(size=1) 
  write(as.character(pop_size_df$pop), glue("/nfs/team205/ed6/data/milo_benchmark/pop_sample_{i}.txt"))
}

Sanity check of synthetic labels on embryo data

pops <- scan("/nfs/team205/ed6/data/milo_benchmark/pop_sample_2_clean.txt", what="")
umap_ls <- lapply(pops, function(p){
  filename <- paste0('/nfs/team205/ed6/data/milo_benchmark/synthetic_data/benchmark_embryo_pop_',
                             str_replace(p, " ", "_"),
                             '_enr0.8_seed45.coldata.csv')
  if (file.exists(filename)){
    coldata <- read_csv(filename)
    colData(embryo_sce) <- column_to_rownames(coldata) %>% DataFrame()
    data.frame(reducedDim(embryo_sce, 'UMAP')) %>%
      dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
      mutate(Condition1_prob=embryo_sce$Condition1_prob > 0.6) %>%
      ggplot(aes(UMAP1, UMAP2, color=Condition1_prob)) +
      ggrastr::geom_point_rast(size=0.1, dpi=700) +
      scale_color_viridis_d() +
      theme_classic(base_size = 16)  
  }
    })

umap_ls

pops <- scan("/nfs/team205/ed6/data/milo_benchmark/pop_sample_2_clean.txt", what="")
umap_ls <- lapply(pops, function(p){
  filename <- paste0('/nfs/team205/ed6/data/milo_benchmark/synthetic_data/benchmark_embryo_pop_',
                             str_replace(p, " ", "_"),
                             '_enr0.9_seed45.coldata.csv')
  if (file.exists(filename)){
    coldata <- read_csv(filename)
    colData(embryo_sce) <- column_to_rownames(coldata) %>% DataFrame()
    data.frame(reducedDim(embryo_sce, 'UMAP')) %>%
      dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
      mutate(Condition1_prob=embryo_sce$Condition1_prob > 0.6) %>%
      ggplot(aes(UMAP1, UMAP2, color=Condition1_prob)) +
      ggrastr::geom_point_rast(size=0.1, dpi=700) +
      scale_color_viridis_d() +
      theme_classic(base_size = 16)  
  }
    })

umap_ls

Pick threshold for cells in true DA regions based on p(C1) distribution

Read full outcomes and true probabilities

read4_prob_hist <- function(data_id){
  outdir <- glue("/nfs/team205/ed6/data/milo_benchmark/{data_id}/")
  res_files <- list.files(outdir, pattern=glue("{data_id}.+batchEffect0.DAresults"))
  res_files_full <- list.files(outdir, pattern=glue("{data_id}.+batchEffect0.DAresults"), full.names = TRUE)
  res_files_full_nomeld <- str_subset(res_files_full, "meld|cydar_batch|louvain_batch|milo_batch", negate = TRUE)
  
  res_ls <- lapply(seq_along(res_files_full_nomeld), function(i){ 
    print(paste("Outcome no. ", i))
    benchmark_df <- read_csv(res_files_full_nomeld[i]) 
    ## Check if conditions were swapped in test
    pred_cor <- benchmark_df %>%
      mutate(pred=factor(pred, levels=c("NegLFC", "NotDA", "PosLFC"), ordered = TRUE)) %>%
      mutate(pred=as.numeric(pred)) %>%
      summarise(cor(pred, true_prob))
    ## Swap outcomes if so
    if (!is.na(pred_cor)) {
      if (pred_cor < - 0.1) {
        benchmark_df <- mutate(benchmark_df, pred = ifelse(pred=="NegLFC", "PosLFC", ifelse(pred=="PosLFC", "NegLFC", "NotDA")))
      }
    }
    benchmark_df
    })
  
  res_df <- res_ls  %>%
    purrr::reduce(bind_rows) %>%
    select(true_prob, pred, method) 
  
  return(res_df)  
}

cluster_res_df <- read4_prob_hist("cluster3x")
linear_res_df <- read4_prob_hist("linear3x")
branch_res_df <- read4_prob_hist("branching3x")
embryo_res_df <- read4_prob_hist("embryo")

Plot distribution of P(C1) in cells identified as DA by different methods

calculate_quantile_thresh <- function(res_df){
  return(quantile(1-res_df[res_df$method=="milo",][["true_prob"]], probs = seq(0,1, by=0.25))["75%"])
}

plot_p_hist <- function(res_df){
  ## Calculate threshold
  thresh <- calculate_quantile_thresh(res_df)
  
  ## Plot histogram of p(C1) for predicted DA cells
  p <- res_df %>%
    dplyr::filter(pred!="NotDA") %>%
    # mutate(color=ifelse(true_prob==0.5, "0.5","0")) %>%
    ggplot(aes(1 - true_prob)) +
    # geom_density(aes(color=pred)) +
    facet_grid(method~pred, scales="free_y", labeller=pl_labeller) +
    geom_histogram(bins=50, aes(fill=pred)) +
    xlab("P(C1)") + ylab("# DA cells") +
    theme_bw(base_size=16) +
    scale_fill_brewer(palette="Set1", name="Predicted effect") +
    geom_vline(xintercept = thresh, linetype=2) 
  p
}

# plot_p_hist(cluster_res_df) + ggtitle("clusters") 
plot_p_hist(linear_res_df) + ggtitle("linear") +
plot_p_hist(branch_res_df) + ggtitle("branching") +
plot_p_hist(embryo_res_df) + ggtitle("embryo") +
  plot_layout(guides="collect") +
  ggsave(paste0(figdir, "/pC1_histograms.pdf"), width=18, height = 6)

Use 75% quantile of P(C1) in all cells to define threshold for true DA

linear_thresh <- calculate_quantile_thresh(linear_res_df)

Warning messages:
1: In readChar(file, size, TRUE) : truncating string with embedded nuls
2: In readChar(file, size, TRUE) : truncating string with embedded nuls

branching_thresh <- calculate_quantile_thresh(branch_res_df)
embryo_thresh <- calculate_quantile_thresh(embryo_res_df)

Benchmarking on simulated data

Define utils

## Util to compute benchmark outcome given a probability threshold for true DA

Warning message:
In system("timedatectl", intern = TRUE) :
  running command 'timedatectl' had status 1

.outcome_by_prob <- function(benchmark_df, da_upper){
  DA_thresh <- 1 - da_upper
  benchmark_df <- benchmark_df %>%
    mutate(true = ifelse(benchmark_df$true_prob < DA_thresh, "NegLFC", ifelse(benchmark_df$true_prob > 1-DA_thresh, "PosLFC", "NotDA"))) 
  if (benchmark_df$method[1]=="meld") {
    benchmark_df <- mutate(benchmark_df, pred = ifelse(Condition2 < DA_thresh, "NegLFC", "NotDA")) 
  }
  ## Check if conditions were swapped in test
  pred_cor <- benchmark_df %>%
    group_by(pred) %>%
    summarise(m=mean(true_prob)) %>%
    ungroup() %>%
    mutate(pred=factor(pred, levels=c("NegLFC", "NotDA", "PosLFC"), ordered = TRUE)) %>%
    mutate(pred=as.numeric(pred)) %>%
    summarise(c=cor(pred, m)) %>%
    pull(c)
  ## Swap outcomes if so
  if (!is.na(pred_cor)) {
    if (pred_cor < 0) {
      benchmark_df <- mutate(benchmark_df, pred = ifelse(pred=="NegLFC", "PosLFC", ifelse(pred=="PosLFC", "NegLFC", "NotDA")))
    }
  }
  calculate_outcome(benchmark_df)
}

## Util to plot benchmarking results on UMAP
plot_outcome_umap <- function(sce, method, pop, enr, seed, batchEffect, true=FALSE, data_id="embryo", 
                              red_dim="umap_batch",
                              rasterize=FALSE){
  coldata_file <- list.files(indir, pattern=paste0(".+",data_id,"_pop_",  pop,"_enr",enr,"_seed", seed, ".coldata.csv"),
                           full.names = TRUE)
  colData(sce) <- read_csv(coldata_file) %>% column_to_rownames() %>% DataFrame()
  
  res_file <- list.files(outdir, 
                         pattern= paste0(".+",data_id,"_pop_", pop,"_enr",enr,"_seed", seed, '.+batchEffect',batchEffect,".DAresults.",method,".csv"),
                         full.names = TRUE)
  if (method=="meld") {
    sce$predicted <- ifelse(read_csv(res_file)[["Condition2"]] < 0.3, "NegLFC", 'NotDA')  
  } else if (method=="daseq") {
    sce$predicted <- ifelse(read_csv(res_file)[["pred"]] == "NegLFC", 'PosLFC', ifelse(read_csv(res_file)[["pred"]] == "PosLFC", "NegLFC", "NotDA"))
  } else {
    sce$predicted <- read_csv(res_file)[["pred"]]
  }
  if (true) {
    pl <- data.frame(reducedDim(sce, red_dim)) %>%
    dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=sce$Condition1_prob) %>%
    arrange(true) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) + 
    geom_point(size=0.1) +
      scale_color_viridis_c(name="C1 probability", limits=c(0.5,0.9)) +
     theme_classic(base_size=18) +
      guides(color=guide_colorbar(title.position="top", title.hjust = 0.5)) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank(),
           legend.position = "top") +
     xlab("UMAP1") + ylab("UMAP2") 
  } else {
    pl <- data.frame(reducedDim(sce, red_dim)) %>%
    dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
    dplyr::mutate(method=method) %>%
    dplyr::mutate(predicted=factor(sce$predicted, levels=c("NegLFC", "PosLFC", 'NotDA'))) %>%
    dplyr::arrange(- predicted) %>%
    ggplot(aes(UMAP1, UMAP2, color=predicted)) + 
    geom_point(size=0.1) +
    geom_point(data= . %>% filter(predicted!="NotDA"), size=0.2) +
     scale_color_manual(values=c(NegLFC="red", NotDA="grey", PosLFC="blue"), 
                        labels=c(NegLFC="Enriched in C1", NotDA="No DA", PosLFC="Depleted in C1"), name='') +
     guides(color = guide_legend(override.aes = list(size=1))) +
     theme_classic(base_size=18) +
     theme(legend.position=c(.15,.2), axis.ticks = element_blank(),
           axis.text = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2") +
      facet_wrap(method~.)  
  }
  if (rasterize) {
    pl <- pl + rasterize(geom_point(size=0.1), dpi = 300)
  }
  pl
}

## Convert condition probability to LFC
prob2FC <- function(prob){
  log((prob)/(1 - prob))
}

Linear - 1D trajectory

outdir <- "/nfs/team205/ed6/data/milo_benchmark/linear3x/"
res_files <- list.files(outdir, pattern="linear.+DAresults")
res_files_full <- list.files(outdir, pattern="linear3x.+DAresults", full.names = TRUE)
in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="linear3x.+coldata.csv")

## Make data frame w benchmark parameters
linear_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect")) 

## Load results
linear_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = linear_thresh) %>%
      mutate(DA_thresh=linear_thresh) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(linear_out_meta_df[i,]) 
  pop_enr <- as.numeric(linear_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows) %>%
  ## Remove MELD
  filter(method!="meld")

linear_outcome_df 

pl_df <- linear_outcome_df %>%
  filter(method!="milo_batch") %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(pop=str_replace(pop, "_", " ")) %>%
  mutate(pop=factor(pop, levels=unique(pop))) %>%
  filter(DA_thresh >= linear_thresh & batchEffect==0) 
pl_right <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
    theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())
pl_left <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
    theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())

pl <- pl_right / pl_left +
  plot_layout(guides='collect') &
    theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())

pl

Plot UMAP

linear_sce <- readRDS("~/data/milo_benchmark/linear3x_data_bm.RDS")
indir <- "~/data/milo_benchmark/synthetic_data/"

names(reducedDims(linear_sce))[2] <- "umap_batch"
colnames(reducedDims(linear_sce)[[2]]) <- c("X1", "X2") 

pl_top <- plot_outcome_umap(linear_sce, method = 'milo', pop="M5", enr = 0.9, seed = 43,batchEffect = 0, true = TRUE, data_id="linear3x") +
  theme(legend.position="top") 

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  group_id = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

plot_linear <- ((pl_top) | (pl & theme(legend.position="top"))) +
  plot_layout(widths = c(0.5,2))
plot_linear

Branching

outdir <- "/nfs/team205/ed6/data/milo_benchmark/branching3x/"
res_files <- list.files(outdir, pattern="branching3x.+DAresults")
res_files_full <- list.files(outdir, pattern="branching3x.+DAresults", full.names = TRUE)
in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="branching3x.+coldata.csv")

## Make data frame w benchmark parameters
branch_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect")) 

## Load results
branch_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = branching_thresh) %>%
      mutate(DA_thresh=branching_thresh) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(branch_out_meta_df[i,]) 
  pop_enr <- as.numeric(branch_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows) %>%
  ## Remove MELD
  filter(method!="meld")

branch_outcome_df 
  

pl_df <- branch_outcome_df %>%
  filter(method!="milo_batch") %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(pop=str_replace(pop, "_", " ")) %>%
  mutate(pop=factor(pop, levels=unique(pop))) %>%
  filter(DA_thresh >= branching_thresh & batchEffect==0) 
pl_right <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
    theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())
pl_left <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
  theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())

pl <- pl_right / pl_left +
plot_layout(guides='collect') &
    theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())

pl

Plot UMAP

branch_sce <- readRDS("~/data/milo_benchmark/branching3x_data_bm.RDS")
indir <- "~/data/milo_benchmark/synthetic_data/"

names(reducedDims(branch_sce))[2] <- "umap_batch"
colnames(reducedDims(branch_sce)[[2]]) <- c("X1", "X2") 

pl_top <- plot_outcome_umap(branch_sce, method = 'milo', pop="M5", enr = 0.9, seed = 43,batchEffect = 0, true = TRUE, data_id="branching3x") +
  theme(legend.position="top") 

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  group_id = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

plot_branch <- ((pl_top) | (pl & theme(legend.position="top"))) +
  plot_layout(widths = c(0.5,2))
plot_branch

Simple clustering

outdir <- "/nfs/team205/ed6/data/milo_benchmark/cluster3x/"
res_files <- list.files(outdir, pattern="cluster3x.+DAresults")
res_files_full <- list.files(outdir, pattern="cluster3x.+DAresults", full.names = TRUE)
in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="cluster3x.+coldata.csv")

## Make data frame w benchmark parameters
cluster_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect")) 

## Load results
cluster_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = 0.55) %>%
      mutate(DA_thresh=0.55) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(cluster_out_meta_df[i,]) 
  pop_enr <- as.numeric(cluster_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows) %>%
  ## Remove MELD
  filter(method!="meld")

[1] "Outcome no.  1"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  2"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  3"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  4"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  5"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  6"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  7"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  8"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  9"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  10"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  11"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  12"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  13"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  14"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  15"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  16"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  17"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  18"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  19"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  20"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  21"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  22"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  23"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  24"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  25"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  26"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  27"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  28"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  29"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  30"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  31"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  32"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  33"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  34"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  35"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  36"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  37"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  38"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  39"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  40"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  41"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  42"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  43"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  44"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  45"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  46"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  47"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  48"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  49"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  50"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  51"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  52"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  53"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  54"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  55"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  56"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  57"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  58"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  59"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  60"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  61"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  62"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  63"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  64"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  65"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  66"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  67"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  68"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  69"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  70"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  71"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  72"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  73"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  74"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  75"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  76"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  77"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  78"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  79"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  80"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  81"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  82"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  83"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  84"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  85"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  86"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  87"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  88"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  89"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  90"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  91"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  92"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  93"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  94"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  95"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  96"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  97"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  98"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  99"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  100"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  101"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  102"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  103"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  104"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  105"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  106"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  107"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  108"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  109"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  110"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  111"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  112"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  113"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  114"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  115"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  116"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  117"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  118"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  119"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  120"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  121"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  122"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  123"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  124"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  125"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  126"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  127"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  128"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  129"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  130"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  131"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  132"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  133"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  134"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  135"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  136"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  137"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  138"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  139"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  140"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  141"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  142"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  143"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  144"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  145"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  146"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  147"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  148"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  149"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  150"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  151"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  152"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  153"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  154"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  155"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  156"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  157"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  158"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  159"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  160"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  161"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  162"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  163"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  164"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  165"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  166"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  167"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  168"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  169"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  170"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  171"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  172"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  173"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  174"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  175"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  176"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  177"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  178"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  179"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  180"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  181"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  182"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  183"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  184"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  185"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  186"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  187"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  188"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  189"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  190"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  191"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  192"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  193"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  194"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  195"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  196"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  197"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  198"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  199"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  200"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  201"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  202"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  203"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  204"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  205"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  206"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  207"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  208"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  209"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  210"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  211"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  212"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  213"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  214"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  215"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  216"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  217"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  218"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  219"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  220"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  221"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  222"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  223"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  224"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  225"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  226"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  227"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  228"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  229"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  230"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Condition2 = col_double(),
  method = col_character(),
  true = col_character(),
  true_prob = col_double()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  231"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  232"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  233"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  234"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  235"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  236"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  237"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  238"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  239"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  240"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  241"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  242"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  243"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

cluster_outcome_df 

NA

pl_df <- cluster_outcome_df %>%
 filter(method!="milo_batch") %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(pop=str_replace(pop, "_", " ")) %>%
  mutate(pop=factor(pop, levels=unique(pop))) %>%
  filter(DA_thresh >= 0.55 & batchEffect==0) 
pl_right <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
  theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
    axis.title.x = element_blank())
pl_left <- pl_df %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
  theme(#axis.text.x = element_blank(), axis.ticks.x = element_blank(), 
        axis.title.x = element_blank()
        )

pl <- pl_right / pl_left +
plot_layout(guides='collect') &
  theme(#axis.text.x=element_blank(), axis.ticks.x=element_blank(), 
        axis.title.x=element_blank()
        )

pl

Plot UMAP

cluster_sce <- readRDS("~/data/milo_benchmark/cluster3x_data_bm.RDS")
indir <- "~/data/milo_benchmark/synthetic_data/"

names(reducedDims(cluster_sce))[2] <- "umap_batch"
colnames(reducedDims(cluster_sce)[[2]]) <- c("X1", "X2") 
pl_top <- plot_outcome_umap(cluster_sce, method = 'milo', pop="B3", enr = 0.9, seed = 43,batchEffect = 0, true = TRUE, data_id="cluster3x") + 
 theme(legend.position="top") 

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

plot_cluster <- ((pl_top) | (pl & theme(legend.position="top"))) +
  plot_layout(widths = c(0.5,2))
plot_cluster

Assemble figure

fig <- (
   (plot_cluster ) /
   (plot_linear & theme(legend.position="none")) /
   (plot_branch & theme(legend.position="none"))
   ) + 
  plot_layout() &
  theme(strip.text = element_text(size=11))

fig +
  ggsave(paste0(figdir, "simData_figure.png"), width=10, height = 13) +
  ggsave(paste0(figdir, "simData_figure.pdf"), width=10, height = 13)

Check some examples

plot_outcome_umap(sce = linear_sce, method="louvain", pop="B3", enr=0.9, seed=43, batchEffect = '0', data_id = "cluster3x", true=FALSE) +
plot_outcome_umap(sce = linear_sce, method="cydar",   pop="B3", enr=0.9, seed=43, batchEffect = '0', data_id = "cluster3x", true=FALSE) +
plot_outcome_umap(sce = linear_sce, method="milo",    pop="B3", enr=0.9, seed=43, batchEffect = '0', data_id = "cluster3x", true=FALSE) +
plot_outcome_umap(sce = linear_sce, method="daseq",   pop="B3", enr=0.9, seed=43, batchEffect = '0', data_id = "cluster3x", true=FALSE) +
plot_outcome_umap(sce = linear_sce, method="daseq", pop="B3", enr=0.9, seed=43, batchEffect = '0', data_id = "cluster3x", true=TRUE) +
  plot_layout(guides="collect")

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

'-' not meaningful for factors
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

'-' not meaningful for factors
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

'-' not meaningful for factors
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

'-' not meaningful for factors
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  rowname = col_character(),
  Block = col_character(),
  Replicate = col_character(),
  cell = col_character(),
  celltype = col_character(),
  synth_labels = col_character(),
  synth_samples = col_character(),
  synth_batches = col_character(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double(),
  true_labels = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

MELD / Milo comparison

Comparing accuracy and precision of the fold-change estimates of the two methods running MELD and Milo on the same simulation on the embryo data.

We estimate LFCs for both methods (see run_miloVSmeld.R) and compare with the true LFC

## Mean squared error
MSE <- function(true, pred){
  mean((true - pred)^2)
}

## Mean bias error
MBE <- function(true, pred){
  mean(true - pred)
}

## Mean abs error
MAE <- function(true, pred){
  mean(abs(true - pred))
}

## std error
SE <- function(true, pred){
  sqrt((sum((true - pred)^2))/(length(pred) - 2))
}

## Convert condition probability to LFC
prob2FC <- function(prob){
  log((prob)/(1 - prob))
}

lfc_comparison <- function(milo_out, meld_out){
  # ## Annotate nhoods that overlap true DA cells
  # milo_out$DAres <- annotateNhoods(milo_out$Milo, milo_out$DAres, coldata_col = "true_labels")
  
  ## Take probability estimate at nhood indices 
  ## (taking the avg of cells in the nhood gives equivalent results)
  # nh_meld <- meld_out[unlist(nhoodIndex(milo_out$Milo)),]
  nh_meld <- meld_out[milo_out$nhoodIndex,]
  
  ## Convert probabilities to LFC  
  lfc_df <- nh_meld %>%
        mutate(milo_LFC=milo_out$logFC,
         milo_signif=milo_out$SpatialFDR < 0.1,
         true_labels=milo_out$true_labels,
         nh_size=milo_out$nh_size
         ) %>%
        mutate(true_LFC=prob2FC(true_prob), meld_LFC=prob2FC(Condition2)) %>%
        select(- method, - true_prob, - Condition2) %>%
    pivot_longer(cols=c(milo_LFC, meld_LFC), names_to="method", values_to="estimated_LFC")
    
  lfc_df
  }

## Load results from run_miloVSmeld.R
data_dir <- '/nfs/team205/ed6/data/milo_benchmark/MELDvsMilo/'
milo_files <- list.files(data_dir, pattern="milo.+csv", full.names = TRUE)
milo_res_outs <- lapply(milo_files, read_csv)

meld_files <- str_remove_all(milo_files, paste0(data_dir, "/")) %>%
  str_replace("milo", "meld") %>%
  paste0(data_dir, .)
meld_outs <- lapply(meld_files, read_csv)

meldVSmilo_meta <- data.frame(file_id = str_remove_all(meld_files, paste0(data_dir, "|/|benchmark_embryo_pop_|.csv"))) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  separate(col = method, sep = "\\.", into=c("method", 'k')) %>%
  mutate(seed=str_remove(seed, "seed"), 
         k=as.numeric(str_remove(k, "k")), 
         batchEffect=str_remove(batchEffect, "batchEffect"),
         ) %>%
  select(- method)
  
lfc_df <- lapply(seq_along(milo_res_outs), function(i) lfc_comparison(milo_res_outs[[i]], meld_outs[[i]]) %>%
                   bind_cols(meldVSmilo_meta[i,])) %>%
  purrr::reduce(bind_rows)

pl_labeller <- labeller(
  enr = enr_labeller2,
  true_labels = label_value,
  pop = label_value,
  .default = label_both
)

pl_milo <- lfc_df %>% 
  filter(pop %in% c("Gut", "Erythroid1", "Somitic_mesoderm")) %>%
  filter(method=="milo_LFC" & batchEffect=='0') %>%
  group_by(enr, true_labels, pop, k) %>%
  mutate(MAE = MAE(true_LFC, estimated_LFC)) %>%
  ggplot(aes(true_LFC, estimated_LFC)) + 
  geom_point(size=0.6, aes(color=true_labels)) +
  geom_hline(yintercept = 0, linetype=2) +
  geom_vline(xintercept = 0, linetype=2) +
  geom_abline(linetype=2) +
  scale_color_manual(values=c(NegLFC="red", NotDA="grey"), labels=c(NegLFC="NegLFC", NotDA="no DA"), name='') +
  guides(color=guide_legend(override.aes = c(size=2))) +
  facet_grid(pop~enr, labeller = pl_labeller) +
  xlab("True LFC") + ylab("Milo estimated LFC") +
  theme_bw(base_size = 14)

pl_meld <- lfc_df %>% 
  filter(pop %in% c("Gut", "Erythroid1", "Somitic_mesoderm")) %>%
  filter(method=="meld_LFC" & batchEffect=='0') %>%
  # filter(enr==0.9 & k==50 & true_labels=="NegLFC")
  group_by(enr, true_labels, pop, k) %>%
  mutate(MAE = MAE(true_LFC, estimated_LFC)) %>%
  ggplot(aes(true_LFC, estimated_LFC)) + 
  geom_point(size=0.6, aes(color=true_labels)) +
  geom_hline(yintercept = 0, linetype=2) +
  geom_vline(xintercept = 0, linetype=2) +
  geom_abline(linetype=2) +
  xlab("True LFC") + ylab("MELD estimated LFC") +
  guides(color=guide_legend(override.aes = c(size=2))) +
  scale_color_manual(values=c(NegLFC="red", NotDA="grey"), labels=c(NegLFC="NegLFC", NotDA="no DA"), name="") +
  facet_grid(pop~enr, labeller = pl_labeller) +
  theme_bw(base_size = 14)

pl_left <- (pl_milo | pl_meld) + 
  plot_layout(guides="collect") &
  theme(legend.position="top", legend.justification = c(1,1))

pl_milo <- lfc_df %>% 
  filter(pop %in% c("Gut", "Erythroid1", "Somitic_mesoderm")) %>%
  filter(method=="milo_LFC" & batchEffect=='0.5') %>%
  group_by(enr, true_labels, pop, k) %>%
  mutate(MAE = MAE(true_LFC, estimated_LFC)) %>%
  ggplot(aes(true_LFC, estimated_LFC)) + 
  geom_point(size=0.6, aes(color=true_labels)) +
  geom_hline(yintercept = 0, linetype=2) +
  geom_vline(xintercept = 0, linetype=2) +
  geom_abline(linetype=2) +
  scale_color_manual(values=c(NegLFC="red", NotDA="grey"), labels=c(NegLFC="NegLFC", NotDA="no DA"), name='') +
  guides(color=guide_legend(override.aes = c(size=2))) +
  facet_grid(pop~enr, labeller = pl_labeller) +
  xlab("True LFC") + ylab("Milo estimated LFC") +
  theme_bw(base_size = 14)

pl_meld <- lfc_df %>% 
  filter(pop %in% c("Gut", "Erythroid1", "Somitic_mesoderm")) %>%
  filter(method=="meld_LFC" & batchEffect=='0.5') %>%
  # filter(enr==0.9 & k==50 & true_labels=="NegLFC")
  group_by(enr, true_labels, pop, k) %>%
  mutate(MAE = MAE(true_LFC, estimated_LFC)) %>%
  ggplot(aes(true_LFC, estimated_LFC)) + 
  geom_point(size=0.6, aes(color=true_labels)) +
  geom_hline(yintercept = 0, linetype=2) +
  geom_vline(xintercept = 0, linetype=2) +
  geom_abline(linetype=2) +
  xlab("True LFC") + ylab("MELD estimated LFC") +
  guides(color=guide_legend(override.aes = c(size=2))) +
  scale_color_manual(values=c(NegLFC="red", NotDA="grey"), labels=c(NegLFC="NegLFC", NotDA="no DA"), name="") +
  facet_grid(pop~enr, labeller = pl_labeller) +
  theme_bw(base_size = 14)


pl_left_be <- (pl_milo | pl_meld) + 
  plot_layout(guides="collect") &
  theme(legend.position="top", legend.justification = c(1,1))

pl_right1 <- lfc_df %>%
  filter(k==50) %>%
  group_by(method, pop, true_labels, k, enr, batchEffect) %>%
  summarise(MSE = MSE(true_LFC, estimated_LFC), sd=sd(abs(true_LFC - estimated_LFC))) %>%
  ungroup() %>%
  mutate(method = str_remove(method,"_LFC")) %>%
  rename(`Batch effect \nmagnitude` = batchEffect) %>%
  filter(true_labels=="NotDA") %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)), 1)), MSE, color = method)) +
  geom_boxplot() +
  facet_grid(true_labels~`Batch effect \nmagnitude`, labeller=pl_labeller) +
  theme_bw(base_size=18) +
  xlab("Simulated log-Fold Change") +
  scale_color_methods() +
  theme(legend.position="top", legend.justification = c(1,1)) +
  ylim(0,2.1)
pl_right2 <- lfc_df %>%
  filter(k==50) %>%
  group_by(method, pop, true_labels, k, enr, batchEffect) %>%
  summarise(MSE = MSE(true_LFC, estimated_LFC), sd=sd(abs(true_LFC - estimated_LFC))) %>%
  ungroup() %>%
  mutate(method = str_remove(method,"_LFC")) %>%
  rename(`Batch effect \nmagnitude` = batchEffect) %>%
  filter(true_labels=="NegLFC") %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)), 1)), MSE, color = method)) +
  geom_boxplot() +
  facet_grid(true_labels~`Batch effect \nmagnitude`, labeller=pl_labeller) +
  theme_bw(base_size=18) +
  xlab("Simulated log-Fold Change") +
  scale_color_methods() +
  theme(legend.position="top", legend.justification = c(1,1)) +
  ylim(0,2.1)

pl_right <- pl_right1 + pl_right2 +
  plot_layout(guides="collect") &
  theme(legend.position="top", legend.justification = c(1,1)) 

(pl_left /pl_left_be /  pl_right) +
  plot_layout(heights=c(1.5, 1.5, 1)) +
  plot_annotation(tag_levels = "A") +
  # ggsave("~/mount/gdrive/milo/Figures/benchmark_v2/SFig_MELDvsMilo.pdf", width = 15, height = 16) +
  ggsave("~/mount/gdrive/milo/Figures/resubmission/supplementary/suppl_fig_meldVSmilo.pdf", width = 15, height = 19) 
  ggsave("~/mount/gdrive/milo/Figures/benchmark_v2/SFig_MELDvsMilo.png", width = 15, height = 16)

Embryo data

W/o batch effect

## Read outputs
outdir <- "/nfs/team205/ed6/data/milo_benchmark/embryo/"
res_files <- list.files(outdir, pattern=".DAresults")
res_files_full <- list.files(outdir, pattern=".DAresults", full.names = TRUE)
# in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="coldata.csv")

## Make data frame w benchmark parameters
out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove_all(batchEffect, "batchEffect")) 

## Read benchmark results w/o batch effects
no_batch_files <- str_detect(res_files_full, "batchEffect0.DAresult") & !str_detect(res_files_full, "louvain_batch|cydar_batch|milo_batch")
# prob_thresh_vec <- seq(0.5, 0.9, 0.05)
outcome_df <- lapply(seq_along(res_files_full[no_batch_files]), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[no_batch_files][i])
  pop_enr <- as.numeric(out_meta_df[no_batch_files,][i,"enr"])
  benchmark_df %>%
      .outcome_by_prob(da_upper = embryo_thresh) %>%
      mutate(DA_thresh=embryo_thresh) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(out_meta_df[no_batch_files,][i,]) %>%
      filter(DA_thresh < as.numeric(enr))
}) %>%
  purrr::reduce(bind_rows)

write_csv(outcome_df, "/nfs/team205/ed6/data/milo_benchmark/outcome_embryo_v3_noBatch.csv")

Read outcome for embryo (built with wrapper build_outcome.R)

outcome_df <- read_csv("/nfs/team205/ed6/data/milo_benchmark/outcome_embryo_v3_noBatch.csv")

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  FN = col_double(),
  TN = col_double(),
  TP = col_double(),
  FP = col_double(),
  TPR = col_double(),
  FPR = col_double(),
  TNR = col_double(),
  FNR = col_double(),
  FDR = col_double(),
  Precision = col_double(),
  Power = col_double(),
  Accuracy = col_double(),
  DA_thresh = col_double(),
  pop = col_character(),
  enr = col_double(),
  seed = col_double(),
  batchEffect = col_double(),
  method = col_character()
)

outcome_df <- outcome_df %>%
  filter(DA_thresh==embryo_thresh) %>%
  filter(pop!="Mixed_mesoderm")

# enr_labeller2 <- function(variable, value, split_lines=FALSE){
#   fc <- round(prob2FC(as.numeric(value)), 2)
#   enr_name <- paste("logFC = ", fc)
#   return(enr_name)
# }

pl_df <- outcome_df %>% 
   filter(batchEffect==0 & str_detect(method, "_batch", negate = TRUE)) %>% 
   mutate(FDR = FP/(TP+FP)) %>% 
   mutate(TPR=ifelse(is.nan(TPR), 0, TPR), 
          FPR=ifelse(is.nan(FPR), 0, FPR), 
          FDR=ifelse(is.nan(FDR), 0, FDR), 
          Precision=ifelse(is.nan(Precision), 0, Precision)) %>% 
   filter(DA_thresh >= embryo_thresh & batchEffect==0)  %>% 
   mutate(enr=as.character(enr))  

p_top <- pl_df %>%
  filter(batchEffect=="0") %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  # geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
  theme(axis.text.x = element_blank(), axis.ticks.x = element_blank(), axis.title.x = element_blank())

p_bottom <- pl_df %>%
  filter(batchEffect=="0") %>%
  ggplot(aes(as.factor(round(prob2FC(as.numeric(enr)),1)), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=pl_labeller) +
  xlab('Simulated log-Fold Change') 

(p_top / p_bottom) + plot_layout(guides="collect") +
  ggsave("~/mount/gdrive/milo/Figures/benchmark_v2/bm_nobatch.pdf",  width = 11, height = 6) +
  ggsave("~/mount/gdrive/milo/Figures/benchmark_v2/bm_nobatch.png",  width = 11, height = 6)

Impact of population size

Range of sizes

mutate(outcome_df, nDA= TP+FN) %>%
  filter(method=="milo" & batchEffect=="0") %>%
  summarise(min(nDA))
  ggplot(aes(nDA,TPR)) +
  geom_point()

Generate labels

• Changing logit param: to have bigger DA region around the same centroid
• capping the enrichment, so to have the same number of DA cells with different fold changes

I pick Erythroid2 and Caudal_neurectoderm as the best and worse performance regions

save_labels_capped <- function(sce, cap_enr, pop, seed, a_logit){
  if (str_detect(pop, "_")) {
    pop <- str_replace(pop, "_", " ")
  }
  sce <- add_synthetic_labels_pop(sce, pop=pop, pop_column = "celltype", seed=seed, pop_enr=0.9, cap_enr = cap_enr,a_logit =  a_logit)
  true_labels <- ifelse(sce$Condition1_prob < (1-cap_enr), "NegLFC", ifelse(sce$Condition2_prob > cap_enr, "PosLFC", "NotDA"))
  colData(sce)[["true_labels"]] <- true_labels
  if (str_detect(pop, " ")) {
    pop <- str_replace(pop, " ", "_")
  }
  
  ## Save coldata
  outdir <-'/nfs/team205/ed6/data/milo_benchmark/synthetic_data/'
  a <- str_remove(as.character(a_logit), "\\.")
  outprefix <- str_c("benchmark_embryo_pop_", pop, '_enr', cap_enr, "_seed", seed, "_capped_logit", a)
  coldata <- data.frame(colData(sce)) %>% rownames_to_column()
  write_csv(coldata, str_c(outdir, outprefix, ".coldata.csv"))
  }


sce <- embryo_sce
## Exclude mixed mesoderm (it's too mixed)
pops <- c("Erythroid2", "Caudal_neurectoderm")

for (p in pops) {
  for (e in seq(0.75,0.95, by=0.1) ) {
    for (s in c(43,44,45) ) {
      for (a in c(0.3,0.5, 0.7) ) {
      save_labels_capped(embryo_sce, e, p, s, a)
      }
    }
  }
}

I run the benchmark with synthetic labels generated above (see submit_benchmark_capped.sh)

Read all results

outdir <- "/nfs/team205/ed6/data/milo_benchmark/capped_logit/"
res_files <- list.files(outdir, pattern=".+DAresults.+.csv")
res_files_full <- list.files(outdir, pattern=".+DAresults.+.csv", full.names = TRUE)
# in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="_enr0.[78].+coldata.csv")

## Make data frame w benchmark parameters
capped_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed",'capped','logit', "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect"),
         logit=str_remove(logit, "logit")) 

## Load results
capped_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = embryo_thresh) %>%
      mutate(DA_thresh=embryo_thresh) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(capped_out_meta_df[i,]) 
  pop_enr <- as.numeric(capped_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows)

outcome_df_popsize <- capped_outcome_df

## Merge with benchmark outcomes
pl_df <- outcome_df_popsize %>%
  filter(str_detect(method, "_batch", negate = TRUE)) %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  # mutate(pop=str_replace(pop, "_", " ")) %>%
  left_join(DA_pop_size_df) %>%
  arrange(pop_size) %>%
  mutate(pop=factor(pop, levels=unique(pop))) %>%
  mutate(enr=as.character(enr))

Joining, by = c("pop", "enr", "seed", "logit")

Explain variability in performance across different populations

Due to low signal to noise ratio for certain simulations

## Read benchmark results w/o batch effects and selecting just top and bottom population
keep_pops <- c("Erythroid2", "Caudal_neurectoderm")
keep_files <- str_detect(res_files_full, "batchEffect0.DAresult") & 
  !str_detect(res_files_full, "louvain_batch|cydar_batch|milo_batch") &
  str_detect(res_files_full, paste0(keep_pops, collapse = "|"))

suppressWarnings({
  outcome_df_2 <- lapply(seq_along(res_files_full[keep_files]), function(i){ 
    print(paste("Outcome no. ", i))
    benchmark_df <- read_csv(res_files_full[keep_files][i])
    pop_enr <- as.numeric(out_meta_df[keep_files,][i,"enr"])
    benchmark_df <- benchmark_df %>%
      mutate(true = ifelse(benchmark_df$true_prob < 1-embryo_thresh, "NegLFC", ifelse(benchmark_df$true_prob > embryo_thresh, "PosLFC", "NotDA"))) 
    
    ## Check if conditions were swapped in test
    pred_cor <- benchmark_df %>%
      group_by(pred) %>%
      summarise(m=mean(true_prob)) %>%
      ungroup() %>%
      mutate(pred=factor(pred, levels=c("NegLFC", "NotDA", "PosLFC"), ordered = TRUE)) %>%
      mutate(pred=as.numeric(pred)) %>%
      summarise(c=cor(pred, m)) %>%
      pull(c)
    ## Swap outcomes if so
    if (!is.na(pred_cor)) {
      if (pred_cor < 0) {
        benchmark_df <- mutate(benchmark_df, pred = ifelse(pred=="NegLFC", "PosLFC", ifelse(pred=="PosLFC", "NegLFC", "NotDA")))
      }
    }
   
    benchmark_df %>%
      mutate(prob_bins=cut(1-true_prob, breaks = seq(0.3, 1, by=0.05), include.lowest = TRUE, right=FALSE)) %>%
        group_by(prob_bins) %>%
        do(calculate_outcome(.)) %>%
        ungroup() %>%
        dplyr::select(- method) %>%
        bind_cols(out_meta_df[no_batch_files,][i,]) 
    }) %>%
    purrr::reduce(bind_rows)
})

[1] "Outcome no.  1"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  2"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  3"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  4"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  5"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  6"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  7"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  8"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  9"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  10"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  11"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  12"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  13"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  14"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  15"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  16"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  17"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  18"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  19"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  20"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  21"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  22"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  23"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  24"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  25"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  26"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  27"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  28"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  29"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  30"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  31"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  32"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  33"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  34"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  35"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  36"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  37"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  38"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  39"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  40"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  41"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  42"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  43"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  44"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  45"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  46"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  47"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  48"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  49"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  50"

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  51"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  52"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  53"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  54"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  55"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  56"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  57"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  58"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  59"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  60"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  61"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  62"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  63"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  64"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  65"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  66"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  67"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  68"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  69"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  70"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  71"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  72"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  73"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  74"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  75"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  76"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  77"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  78"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  79"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  80"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  81"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  82"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  83"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  84"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  85"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  86"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  87"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  88"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  89"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  90"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  91"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  92"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  93"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  94"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  95"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  96"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  97"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  98"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  99"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  100"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  101"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  102"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  103"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  104"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  105"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  106"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  107"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  108"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  109"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  110"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  111"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  112"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  113"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  114"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  115"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  116"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  117"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  118"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  119"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  120"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  121"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  122"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  123"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  124"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  125"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  126"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  127"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  128"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  129"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  130"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  131"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  132"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  133"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  134"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  135"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  136"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  137"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  138"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  139"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  140"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  141"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  142"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  143"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

[1] "Outcome no.  144"

── Column specification ────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.
`summarise()` has grouped output by 'method'. You can override using the `.groups` argument.

outcome_df_2

pl_signal2noise <- function(mypop, myenr, myseed, mymethod){
    if (mymethod=="daseq") {
    pos_outcome <- "PosLFC"
  } else {
    pos_outcome <- "NegLFC"
  }
  popTPR <- outcome_df %>% filter(pop==mypop & enr==myenr & seed==myseed & method==mymethod) %>% pull(TPR)
  res_file <- list.files(outdir, 
                           pattern= paste0(".+",'embryo',"_pop_", mypop, "_enr",myenr,"_seed", myseed, '.+batchEffect0',".DAresults.",mymethod,".csv"),
                           full.names = TRUE)
  
  pl <- read_csv(res_file) %>%
    mutate(outcome = case_when(pred=="NotDA" & true_prob < 1-embryo_thresh ~ "FN",
                          pred==pos_outcome & true_prob < 1-embryo_thresh ~ "TP",
                          pred=="NotDA" & true_prob > 1-embryo_thresh ~ "TN"
                          )) %>%
    filter(outcome %in% c('TP', "FN")) %>%
    ggplot(aes(1 - true_prob, color=outcome)) +
    geom_density(size=1) +
    xlab("P(C1)") +
    ggtitle(paste("TPR = ", round(popTPR,2))) +
    theme_bw(base_size = 14)
  pl
}

pl_top <- wrap_plots(lapply(c(0.75, 0.85, 0.95), function(x) pl_signal2noise("Erythroid2", x, myseed = 43, mymethod = "milo")), ncol=3)

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

pl_bot <- wrap_plots(lapply(c(0.75, 0.85, 0.95), function(x) pl_signal2noise("Caudal_neurectoderm", x, myseed = 43, mymethod = "milo")), ncol=3)

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)


── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  true_prob = col_double(),
  true = col_character(),
  method = col_character(),
  pred = col_character()
)

pl_B <- (pl_top / pl_bot) +
  plot_layout(guides = "collect") 

pl_B +
  ggsave(paste0(figdir,"Sfig_signal2noise.pdf"), height = 6, width = 9) +
  ggsave(paste0(figdir,"Sfig_signal2noise.png"), height = 6, width = 9)

Benchmark with batch effects

Read outcome for embryo (built with wrapper build_outcome.R)

outcome_df <- read_csv("/nfs/team205/ed6/data/milo_benchmark/outcome_embryo_v3_noBatch.csv")

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  FN = col_double(),
  TN = col_double(),
  TP = col_double(),
  FP = col_double(),
  TPR = col_double(),
  FPR = col_double(),
  TNR = col_double(),
  FNR = col_double(),
  FDR = col_double(),
  Precision = col_double(),
  Power = col_double(),
  Accuracy = col_double(),
  DA_thresh = col_double(),
  pop = col_character(),
  enr = col_double(),
  seed = col_double(),
  batchEffect = col_double(),
  method = col_character()
)

Read outcomes (made with wrapper build_outcome.R)

outcome_df_be <- read_csv("/nfs/team205/ed6/data/milo_benchmark/outcome_embryo_v3_wBatch.csv") %>%
  filter(pop!="Mixed_mesoderm")

Missing column names filled in: 'X19' [19]
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  FN = col_double(),
  TN = col_double(),
  TP = col_double(),
  FP = col_double(),
  TPR = col_double(),
  FPR = col_double(),
  TNR = col_double(),
  FNR = col_double(),
  FDR = col_double(),
  Precision = col_double(),
  Power = col_double(),
  Accuracy = col_double(),
  DA_thresh = col_double(),
  pop = col_character(),
  enr = col_double(),
  seed = col_double(),
  batchEffect = col_double(),
  method = col_character(),
  X19 = col_logical()
)

outcome_df <- bind_rows(outcome_df, outcome_df_be)

batch_labeller <- as_labeller(c(milo="Milo\n(~ batch + condition)", louvain="Louvain\n(~ batch + condition)", cydar="Cydar", daseq='DAseq'))

pl_df <- outcome_df %>%
  filter(method != "milo" & method != "louvain" & method != "cydar_batch") %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
          FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(method=str_remove(method,"_batch")) %>%
  filter(enr >= 0.8) %>%
  mutate(enr=as.character(enr)) 
  

p_top <- pl_df %>%
  ggplot(aes(as.factor(batchEffect), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(.~method, labeller=batch_labeller) +
  theme(axis.text.x = element_blank(), axis.ticks.x = element_blank(), axis.title.x = element_blank())
  
p_bottom <- pl_df %>%
  ggplot(aes(as.factor(batchEffect), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  xlab("Batch effect magnitude") +
  facet_grid(.~method, labeller=batch_labeller)

(p_top / p_bottom) + plot_layout(guides="collect") +
  ggsave(paste0(figdir, "main_bm_batch.pdf"), width = 11, height = 6) +
  ggsave(paste0(figdir, "main_bm_batch.png"), width = 11, height = 6)

Break down by logFC

pl_df <- outcome_df %>%
  filter(method != "milo" & method != "louvain" & method != "cydar_batch") %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
          FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(method=str_remove(method,"_batch")) %>%
  mutate(enr=as.character(enr)) 

p_top <- pl_df %>%
  ggplot(aes(as.factor(batchEffect), TPR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  facet_grid(enr~method, labeller=labeller(enr=enr_labeller2, method=batch_labeller)) +
  xlab("Batch effect magnitude") 
  
p_bottom <- pl_df %>%
  ggplot(aes(as.factor(batchEffect), FDR, color=method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  scale_fill_manual(values = method_colors, labels=setNames(method_labels, method_names)) +
  xlab("Batch effect magnitude") +
  facet_grid(enr~method, labeller=labeller(enr=enr_labeller2, method=batch_labeller))

(p_top | p_bottom) + plot_layout(guides="collect") + plot_annotation(tag_levels = c("A")) +
  # ggsave(paste0(figdir, "suppl_bm_batch_logFC.pdf"), width = 15, height = 10) +
  # ggsave(paste0(figdir, "suppl_bm_batch_logFC.png"), width = 15, height = 10)
  ggsave(paste0("~/mount/gdrive/milo/Figures/resubmission2/supplementary/suppl_fig_bm_batch.pdf"), width = 18, height = 10)

W and w/o batch control

pl_df <- outcome_df %>%
  filter(method %in% c("milo", "milo_batch")) %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
          FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(design = ifelse(str_detect(method,"_batch"), "~ batch + condition", "~ condition")) %>%
  filter(enr >= 0.8) %>%
  mutate(enr=as.character(enr))

metric='TPR'
pl_left <-pl_df %>%
  mutate(group=paste(method, design)) %>%
  mutate(method=str_remove(method, "_batch")) %>%
  ggplot(aes(as.factor(batchEffect), TPR, color =method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5)  +
  facet_grid(.~design) +
  theme_bw(base_size=16) +
  geom_hline(yintercept = 0.8, linetype=2) +
  # scale_color_manual(values = c(`~ condition`=method_colors["milo"], "black")) 
  scale_color_manual(values = method_colors, labels=c(milo="Milo", louvain="Louvain", cydar="Cydar"), name="") +
  theme(legend.position = "top", legend.justification = c(1,1)) +
  theme(axis.text.x = element_blank(), axis.ticks.x = element_blank(), axis.title.x = element_blank())

metric = "FDR"
pl_right <- 
  pl_df %>%
  mutate(group=paste(method, design)) %>%
  mutate(method=str_remove(method, "_batch")) %>%
  ggplot(aes(as.factor(batchEffect), FDR, color =method)) +
  geom_boxplot(outlier.alpha = 0) +
  ggbeeswarm::geom_quasirandom(alpha=0.5, position="stack", size=0.5)  +
  facet_grid(.~design) +
  xlab("Batch effect magnitude") +
  theme_bw(base_size=16) +
  geom_hline(yintercept = 0.1, linetype=2) +
  # scale_color_manual(values = c(`~ condition`=method_colors["milo"], "black")) 
  scale_color_manual(values = method_colors, labels=c(milo="Milo", louvain="Louvain", cydar="Cydar"), name="") +
  theme(legend.position = "top", legend.justification = c(1,1))


design_plot <- 
  (pl_left / pl_right) +
  plot_layout(guides="collect") &
  theme(legend.position = "top", legend.justification = c(1,1)) 

design_plot +
  ggsave(paste0(figdir, "batchEffect_design.png"), height = 6, width = 10) +
  ggsave(paste0(figdir, "batchEffect_design.pdf"), height = 6, width = 10, useDingbats=FALSE)

NA
NA

Plot examples

Visualize outcome in UMAP

  colData(embryo_sce) <- read_csv(coldata_file) %>% column_to_rownames() %>% DataFrame()

── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
cols(
  .default = col_character(),
  sample = col_double(),
  pool = col_double(),
  sequencing.batch = col_double(),
  doub.density = col_double(),
  doublet = col_logical(),
  cluster = col_double(),
  cluster.sub = col_double(),
  cluster.stage = col_double(),
  cluster.theiler = col_double(),
  stripped = col_logical(),
  sizeFactor = col_double(),
  Condition1_prob = col_double(),
  Condition2_prob = col_double()
)
ℹ Use `spec()` for the full column specifications.

Plot benchmark design

p1 <- umap_bath_df %>%
    dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
    ggplot(aes(UMAP1, UMAP2)) + 
    geom_point_rast(size=0.1, raster.dpi = 500) +
    theme_classic(base_size=16) +
      guides(color=guide_colorbar(title.position="top", title.hjust = 0.5)) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank(),
           legend.position = "top") +
     xlab("UMAP1") + ylab("UMAP2") +
    rasterize(geom_point(size=0.1,  color="grey"), dpi = 300)

p2 <- umap_bath_df %>%
    dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=sce$Condition1_prob) %>%
    mutate(true=ifelse(true < 0.5, 0.5, true)) %>%
    arrange(true) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) + 
    geom_point_rast(size=0.1, raster.dpi = 500) +
      scale_color_viridis_c(name="C1 probability") +
    theme_classic(base_size=14) +
      guides(color=guide_colorbar(title.position="left", title.hjust = 1, title.vjust = 1, barwidth = 0.5, barheight = 3)) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank(),
           legend.position = c(0.75,0.85), legend.background = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2") 
  rasterize(geom_point(size=0.1), dpi = 300)

geom_point: na.rm = FALSE
stat_identity: na.rm = FALSE
position_identity 

p3 <- umap_bath_df %>%
    dplyr::rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true_labels=sce$synth_labels) %>%
    ggplot(aes(UMAP1, UMAP2, color=true_labels)) + 
    geom_point_rast(size=0.1, raster.dpi = 500) +
    scale_color_brewer(palette="Set1", labels=c(Condition1='C1', Condition2="C2"), name="") +
    theme_classic(base_size=16) +
     guides(color=guide_legend(override.aes = list(size=2))) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank(),
           legend.position = c(0.85,0.95), legend.background = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2") 

(p1 + p2 + p3) 

Design matrix

Plot batch effect

library(ggrastr)
be_sce_ls <- lapply(c('0',"0.25", "0.5", "0.75", "1"), function(x) read_input_data(embryo_sce, indir, pop, enr, seed, x))

method="milo"

pop
## Plot condition probability 
be_pl_ls <- lapply(be_sce_ls, function(sce) {
  coldata_file <- list.files(indir, pattern=paste0(pop,".+", "enr",enr,".+",'seed', seed, ".coldata.csv"),
                           full.names = TRUE)
  colData(sce) <- DataFrame(read_csv(coldata_file) %>% column_to_rownames())
  data.frame(reducedDim(sce, "umap_batch")) %>%
rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=sce$Condition1_prob) %>%
    mutate(true=ifelse(true < 0.5, 0.5, true)) %>%
    arrange(true) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) + 
    geom_point_rast(size=0.1, raster.dpi = 500) +
      scale_color_viridis_c(name="C1 probability") +
    theme_classic(base_size=14) +
      guides(color=guide_colorbar(title.position="left", title.hjust = 1, title.vjust = 1, barwidth = 0.5, barheight = 3)) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank(),
           legend.position = c(0.75,0.85), legend.background = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2") 
  }
)


## Plot batch
be_pl_ls2 <- lapply(be_sce_ls, function(sce) {
  coldata_file <- list.files(indir, pattern=paste0(pop,".+", "enr",enr,".+",'seed', seed, ".coldata.csv"),
                           full.names = TRUE)
  colData(sce) <- DataFrame(read_csv(coldata_file) %>% column_to_rownames())
  data.frame(reducedDim(sce, "umap_batch")) %>%
    rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=sce$synth_batches) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) +
    geom_point_rast(size=0.1, raster.dpi = 500) +
    scale_color_viridis_d(name="Batch", option="cividis") +
     theme_classic(base_size=18) +
      guides(color=guide_legend(override.aes = list(size=2))) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2") 
  }
)

ptop <- wrap_plots(be_pl_ls, ncol=1, guides="collect") &
  theme(legend.position = "right") 
pbottom <- wrap_plots(be_pl_ls2, ncol=1, guides="collect")

(ptop | pbottom) +
  plot_layout(guides="collect") +
  ggsave(paste0(figdir, "batchEffect_UMAPs.pdf"), height=13, width = 8) +
  ggsave(paste0(figdir, "batchEffect_UMAPs.png"), height=13, width = 8)

Design matrix w batch effect

m1 <- data.frame(colData(sce)[,c("synth_labels", "synth_samples", 'synth_batches')]) %>%
  distinct(synth_labels,synth_samples) %>%
  mutate(synth_labels=str_remove(synth_labels,"ondition"),
         count=1) %>%
  pivot_wider(id_cols = synth_samples, names_from=synth_labels, values_from=count,values_fill=0) %>%
  pivot_longer(cols=c(C1, C2), names_to="synth_labels") %>%
  ggplot(aes(synth_labels, synth_samples)) +
  geom_tile(fill="white", color="black", size=1) +
  geom_tile(data=. %>% filter(value>0), aes(fill=synth_labels), color="black", size=1) +
  theme_classic(base_size=16)   +
  scale_fill_brewer(palette="Set1") +
  guides(fill='none') +
  xlab("Synthetic \ncondition") + ylab('Synthetic samples') +
  theme(axis.text.y = element_blank(),axis.ticks.y = element_blank(),
        axis.line = element_blank())

m2 <- data.frame(colData(sce)[,c("synth_labels", "synth_samples", 'synth_batches')]) %>%
  distinct(synth_labels,synth_samples, synth_batches) %>%
  mutate(synth_labels=str_remove(synth_labels,"ondition"),
         count=1) %>%
  pivot_wider(id_cols = synth_samples, names_from=synth_batches, values_from=count,values_fill=0) %>%
  pivot_longer(cols=c(B1, B2), names_to="synth_batches") %>%
  ggplot(aes(synth_batches, synth_samples)) +
  geom_tile(fill="white", color="black", size=1) +
  geom_tile(data=. %>% filter(value>0), aes(fill=synth_batches), color="black", size=1) +
  theme_classic(base_size=16)   +
  scale_fill_viridis_d(option="cividis") +
  guides(fill='none') +
  xlab("Synthetic \nbatch") + ylab('Synthetic samples') +
  theme(axis.text.y = element_blank(),axis.ticks.y = element_blank(), axis.title.y = element_blank(),
        axis.line = element_blank())

(m1 + m2) +
# (p1 + p2 + p3) + des_pl +
  plot_layout(widths=c(1,1), ncol=2) +
  ggsave(paste0(figdir, "bm_design_batch.pdf"), height = 4, width = 4)

Correcting batch effect

Load simulations with batch effects and correct with MNN

library(batchelor)

mnn_correct_batch_effect <- function(be_sce, k=50){
  ## Split in two SCE objects
  b1_be_sce <- be_sce[,be_sce$synth_batches=="B1"]
  b2_be_sce <- be_sce[,be_sce$synth_batches=="B2"]
  
  be_mnn <- reducedMNN(reducedDim(b1_be_sce, "pca.corrected"), reducedDim(b2_be_sce, "pca.corrected"), k=k)
  reducedDim(be_sce, "pca.MNN") <- be_mnn$corrected[colnames(be_sce),]
  be_sce
  }

## Simulate batch effects of different magnitude
save_corrected_be <- function(pop, pop_enr, seed, be_sd){
  ## Load coldata and PCA
  outdir <- '/nfs/team205/ed6/data/milo_benchmark/synthetic_data/'
  outprefix <- str_c("benchmark_embryo_pop_", pop, '_enr', pop_enr, "_seed", seed)
  coldata <- read_csv(paste0(outdir, outprefix, ".coldata.csv")) %>% column_to_rownames()
  X_pca <-read_csv(str_c(outdir, outprefix, "_batchEffect", be_sd, ".pca.csv")) %>% column_to_rownames()
  
  ## Add reduced dim + coldata to sce
  colData(embryo_sce) <- DataFrame(coldata)
  reducedDim(embryo_sce, "pca_batch") <- as.matrix(X_pca)
  
  set.seed(seed)
  sce_be <- mnn_correct_batch_effect(embryo_sce)
  X_pca <- reducedDim(sce_be, "pca.MNN")
  ## Save reduced dims
  write_csv(as.data.frame(X_pca) %>% rownames_to_column(), str_c(outdir, outprefix, "_batchEffect", be_sd,".MNNcorrected.pca.csv"))
  }

be_sds <- unique(out_meta_df$batchEffect)
pops <- unique(out_meta_df$pop)
seeds=c(43)

for (pop in pops) {
    for (seed in seeds) {
      for (be_sd in c(0.5, 0.75, 1)) {
      pop_enr = 0.8
      save_corrected_be(pop, pop_enr, seed, be_sd)
      }
    }
  }

Read all results (runned in wrapper submit_)

outdir <- "/nfs/team205/ed6/data/milo_benchmark/MNN_corrected/"
res_files <- list.files(outdir, pattern=".+DAresults.+.csv")
res_files_full <- list.files(outdir, pattern=".+DAresults.+.csv", full.names = TRUE)
# in_files <- list.files("~/data/milo_benchmark/synthetic_data/", pattern="_enr0.[78].+coldata.csv")

## Make data frame w benchmark parameters
mnn_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect")) 

## Load results
mnn_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = embryo_thresh) %>%
      mutate(DA_thresh=embryo_thresh) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(mnn_out_meta_df[i,]) 
  pop_enr <- as.numeric(mnn_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows)

no_be <- outcome_df %>%
  filter(batchEffect=="0") %>%
  mutate(FDR = FP/(TP+FP)) 

uncorrected_be <- outcome_df %>%
  filter(batchEffect %in% c("0.5", "0.75", '1')) %>%
  mutate(FDR = FP/(TP+FP)) 

pl_df <- mnn_outcome_df %>%
  filter(enr > 0.6) %>%
   mutate(batchEffect = str_remove(batchEffect, ".MNNcorrected")) %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(pop=str_replace(pop, "_", " ")) %>%
  # mutate(pop_size=pop_sizes[pop])  %>%
  # arrange(pop_size) %>%
  mutate(pop=factor(pop, levels=unique(pop))) %>%
  mutate(seed=as.numeric(seed), enr=as.numeric(enr), batchEffect=as.numeric(batchEffect))

pl_top <- bind_rows(mutate(no_be, class="no Batch Effect"),
  mutate(uncorrected_be, class="Uncorrected"),
  mutate(pl_df, class="MNN corrected")) %>%
  mutate(class=factor(class, levels=c("no Batch Effect", "MNN corrected", "Uncorrected"))) %>%
    mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  filter(method != "milo_batch" & method !="louvain_batch" & method !="cydar_batch") %>%
  ggplot(aes(as.factor(batchEffect), color=method, TPR)) + 
  geom_boxplot(outlier.size = 0.5) +
  facet_grid(.~class, space="free", scales="free") +
  scale_color_methods() +
  scale_fill_methods() +
  theme_bw(base_size=16) +
  xlab("Batch effect magnitude") +
  theme(axis.text.x = element_blank(), axis.ticks.x = element_blank(), axis.title.x = element_blank())

pl_bottom <- bind_rows(mutate(no_be, class="no Batch Effect"),
  mutate(uncorrected_be, class="Uncorrected"),
  mutate(pl_df, class="MNN corrected")) %>%
  mutate(class=factor(class, levels=c("no Batch Effect", "MNN corrected", "Uncorrected"))) %>%
    mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  filter(method != "milo_batch" & method !="louvain_batch" & method !="cydar_batch") %>%
  ggplot(aes(as.factor(batchEffect), color=method, FDR)) + 
  geom_boxplot(outlier.size = 0.5) +
  facet_grid(.~class, space="free", scales="free") +
  scale_color_methods() +
  scale_fill_methods() +
  theme_bw(base_size=16) +
  xlab("Batch effect magnitude")

mnn_plot <- (pl_top / pl_bottom +
  plot_layout(guides="collect") &
  theme(legend.position="top")) +
  ggsave(paste0("~/mount/gdrive/milo/Figures/resubmission2/supplementary/suppl_fig_bm_MNN.pdf"), height = 6, width = 8) 
  
mnn_plot

(mnn_plot & theme(legend.position = "top")) + 
  # ggsave(paste0(figdir, "batchEffect_MNNcorrected.png"), height = 6, width = 10) +
  # ggsave(paste0(figdir, "batchEffect_MNNcorrected.pdf"), height = 6, width = 10) 
ggsave("~/mount/gdrive/milo/Figures/resubmission/supplementary/suppl_fig_bm_MNN.pdf", height = 6, width = 10)

Plot UMAP

X_pca_mnn <- read_csv("/nfs/team205/ed6/data/milo_benchmark/synthetic_data/benchmark_embryo_pop_Somitic_mesoderm_enr0.8_seed43_batchEffect0.75.MNNcorrected.pca.csv") %>%
  column_to_rownames()

all(rownames(X_pca_mnn) == colnames(embryo_sce))
coldata <- read_csv("/nfs/team205/ed6/data/milo_benchmark/synthetic_data/benchmark_embryo_pop_Somitic_mesoderm_enr0.8_seed43.coldata.csv")
colData(embryo_sce) <- coldata %>% column_to_rownames() %>% DataFrame()
reducedDim(embryo_sce, "pca_batch") <- as.matrix(X_pca_mnn)
embryo_sce <- runUMAP(embryo_sce, dimred="pca_batch", name="umap_batch")

indir <- "~/data/milo_benchmark/synthetic_data/"
res <- read_csv("/nfs/team205/ed6/data/milo_benchmark/benchmark_embryo_pop_Somitic_mesoderm_enr0.8_seed43_batchEffect0.75.DAresults.milo.csv")
embryo_sce$pred <- res$pred
embryo_sce$true <- res$true
plotReducedDim(embryo_sce, "umap_batch", colour_by="Condition1_prob", point_size=0.1)
plotReducedDim(embryo_sce, "umap_batch", colour_by="pred", point_size=0.1)
plotReducedDim(embryo_sce, "umap_batch", colour_by="true", point_size=0.1)

res %>%
  .outcome_by_prob(da_upper = 0.6) %>%
  mutate(FDR = FP/(TP+FP)) 

Show why Milo has TPR=0 if batch covariate not included.

data_id="embryo"
pop="Gut"
pop_enr="0.8"
seed='43'

## Load coldata and PCA
outdir <- '/nfs/team205/ed6/data/milo_benchmark/synthetic_data/'
outprefix <- str_c("benchmark_", data_id, "_pop_", pop, '_enr', pop_enr, "_seed", seed)
coldata <- read_csv(paste0(outdir, outprefix, ".coldata.csv")) %>% column_to_rownames()
X_pca <-read_csv(str_c(outdir, str_remove(outprefix, "_capped_logit.."), "_batchEffect", be_sd, ".pca.csv")) %>% column_to_rownames()  

## Find DA probability x cell
k=50
bm_params = list(
  milo = list(k=k)
  )


## Add reduced dim + coldata to sce
colData(embryo_sce) <- DataFrame(coldata)
reducedDim(embryo_sce, "pca_batch") <- as.matrix(X_pca)

## Run milo
milo_res <- run_milo(embryo_sce, condition_col="synth_labels", sample_col='synth_samples',
                       reduced.dim = "pca_batch", d=30, k=bm_params$milo$k)

design_df <- as.tibble(colData(embryo_sce)[c('synth_samples', "synth_labels")]) %>%
    distinct() %>%
    column_to_rownames('synth_samples')

plotNhoodMA(milo_res$DAres)

ggplot(milo_res$DAres, aes(PValue)) + geom_histogram()

out <- milo2output(milo_res$Milo, milo_res$DAres, out_type = out_type)
head(nhoodCounts(milo_res$Milo)[,design_df$synth_labels=="Condition1"])

Non-linear batch effect

add_batch_effect_nonlinear <- function(sce, batch_col="synth_batches", theta_deg=20, dims=1:2){
  ## Simulate rotation of first 2 PCs
  theta=theta_deg*pi/180
  rotM=matrix(0,nrow=2,ncol=2)
  rotM[1,]=c(cos(theta),-sin(theta))
  rotM[2,]=c(sin(theta),cos(theta))
  
  cellids_sample <- split(sce$cell, sce[[batch_col]])
  X_pca <- reducedDim(sce, "pca.corrected")
  X_pca_batch <- X_pca
  ## Apply rotation to one of the batches
  X_pca_batch[cellids_sample[[names(cellids_sample)[1]]],dims] <- t(rotM %*% t(X_pca_batch[cellids_sample[[names(cellids_sample)[1]]],dims]))
  reducedDim(sce, "pca_batch") <- X_pca_batch
  sce  
  }

coldata_file <- list.files(indir, pattern=paste0(pop,".+", "enr",enr,".+",'seed', seed, ".coldata.csv"),
                           full.names = TRUE)
colData(embryo_sce) <- read_csv(coldata_file) %>% column_to_rownames() %>% DataFrame()
  
batch_col="synth_batches"

embryo_sce <- add_batch_effect_nonlinear(embryo_sce, dims=c(3,5), theta_deg = 30)
embryo_sce <- runUMAP(embryo_sce, dimred="pca_batch", name = 'umap_batch', n_dimred=1:30)  

data.frame(reducedDim(embryo_sce, "umap_batch")) %>%
    rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=embryo_sce$synth_batches) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) +
    geom_point(size=0.1) +
    scale_color_viridis_d(name="Batch", option="cividis") +
     theme_classic(base_size=18) +
      guides(color=guide_legend(override.aes = list(size=2))) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2")

data.frame(reducedDim(embryo_sce, "UMAP")) %>%
    rename(UMAP1=X1, UMAP2=X2) %>%
    mutate(true=embryo_sce$synth_batches) %>%
    ggplot(aes(UMAP1, UMAP2, color=true)) +
    geom_point(size=0.1) +
    scale_color_viridis_d(name="Batch", option="cividis") +
     theme_classic(base_size=18) +
      guides(color=guide_legend(override.aes = list(size=2))) +
     theme(axis.ticks = element_blank(),
           axis.text = element_blank()) +
     xlab("UMAP1") + ylab("UMAP2")


# be_sce_ls <- lapply(c('0',"0.25", "0.5", "0.75", "1"), function(x) read_input_data(embryo_sce, indir, pop, enr, seed, x, run_umap = FALSE))

Make non-linear batch effect for all populations

## Simulate batch effects of different magnitude
save_nonlinear_be <- function(sce, pop, pop_enr, seed){
  ## Load coldata and PCA
  outdir <- '/nfs/team205/ed6/data/milo_benchmark/synthetic_data/'
  outprefix <- str_c("benchmark_embryo_pop_", pop, '_enr', pop_enr, "_seed", seed)
  coldata <- read_csv(paste0(outdir, outprefix, ".coldata.csv")) %>% column_to_rownames()
  ## Add reduced dim + coldata to sce
  colData(sce) <- DataFrame(coldata)
  
  set.seed(seed)
  sce_be <- add_batch_effect_nonlinear(embryo_sce, dims=c(3,5), theta_deg = 30)
  X_pca <- reducedDim(sce_be, "pca_batch")
  ## Save reduced dims
  write_csv(as.data.frame(X_pca) %>% rownames_to_column(), str_c(outdir, outprefix, "_batchEffectNonLinear30.pca.csv"))
  }

pops <- unique(out_meta_df$pop)
pop_enrs <- unique(out_meta_df$enr)
seeds <- unique(out_meta_df$seed)

for (pop in pops) {
  for (pop_enr in pop_enrs[1:3]) {
    for (seed in seeds) {
      save_nonlinear_be(embryo_sce, pop, pop_enr, seed)
    }
  }
}

Running in run_DA_nonlinear.R.

Read all results

outdir <- "/nfs/team205/ed6/data/milo_benchmark/"
res_files <- list.files(outdir, pattern="NonLinear.+DAresults.+.csv")
res_files_full <- list.files(outdir, pattern="NonLinear.+DAresults.+.csv", full.names = TRUE)

## Make data frame w benchmark parameters
nonl_out_meta_df <- data.frame(file_id = str_remove_all(res_files, "benchmark_embryo_pop_|.csv")) %>%
  separate(col = file_id, sep = ".DAresults.", into=c("file_id", "method")) %>%
  separate(col = file_id, sep = "_enr", into=c("pop", "file_id")) %>%
  separate(col = file_id, sep = "_", into=c("enr", "seed", "batchEffect")) %>%
  mutate(seed=str_remove(seed, "seed"), batchEffect=str_remove(batchEffect, "batchEffect"))

## Load results
nonl_outcome_df <- lapply(seq_along(res_files_full), function(i){ 
  print(paste("Outcome no. ", i))
  benchmark_df <- read_csv(res_files_full[i]) %>%
      .outcome_by_prob(da_upper = 0.6) %>%
      mutate(DA_thresh=0.6) %>%
      ungroup() %>%
      dplyr::select(- method) %>%
      bind_cols(nonl_out_meta_df[i,]) 
  pop_enr <- as.numeric(nonl_out_meta_df[i,"enr"])
  benchmark_df
  }) %>%
  purrr::reduce(bind_rows)

nonl_outcome_df 

pl_df <- nonl_outcome_df %>%
  filter(enr > 0.6) %>%
  mutate(FDR = FP/(TP+FP)) %>%
  mutate(TPR=ifelse(is.nan(TPR), 0, TPR),
         FPR=ifelse(is.nan(FPR), 0, FPR),
         FDR=ifelse(is.nan(FDR), 0, FDR),
         Precision=ifelse(is.nan(Precision), 0, Precision)) %>%
  mutate(pop=str_replace(pop, "_", " ")) 

pl_df %>%
  ggplot(aes(pop, TPR)) + 
  ggbeeswarm::geom_quasirandom(alpha=0.5) +
  geom_hline(yintercept = 0.8, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_brewer(palette="Dark2") +
  facet_grid(enr~., labeller = enr_labeller) +
pl_df %>%
  ggplot(aes(pop, FDR)) + 
  ggbeeswarm::geom_quasirandom(alpha=0.5) +
  geom_hline(yintercept = 0.1, linetype=2) +
  theme_bw(base_size = 16) +
  scale_color_brewer(palette="Dark2") +
  facet_grid(enr~., labeller = enr_labeller) +
plot_layout(guides='collect') 

lin_be <- outcome_df %>%
  filter(method=="milo_batch" & enr %in% c(0.7, 0.8) & DA_thresh==0.6)

bind_rows(
  mutate(lin_be, class="Linear batch effect"),
  mutate(nonl_outcome_df, class="Non Linear\n batch effect") %>%
    filter(enr > 0.6) %>%
    mutate(enr=as.numeric(enr), seed=as.numeric(seed), batchEffect=as.numeric(str_remove(batchEffect, "NonLinear"))) 
  ) %>%
  ggplot(aes(as.factor(batchEffect), TPR, color=method)) +
  geom_boxplot() +
  ggbeeswarm::geom_quasirandom(alpha=0.5) +
  facet_grid(enr~class, scales="free", space="free") +
  scale_color_manual(values=method_colors[[4]], labels=setNames(method_labels, method_names)) +
  theme_bw(base_size=16) +
  xlab("Batch effect magnitude")

Louvain on iterative subclustering

plotUMAP(embryo_sce, colour_by="Condition1_prob")

Subcluster all celltype populations (there is a caveat here that I am not recalculating the PCA space for all subsets, but it should bring home the message regardless)

embryo_sce$subcluster <- NA
k=15

for (pop in unique(embryo_sce$celltype)) {
  pop_sce <- embryo_sce[,embryo_sce$celltype==pop]
  X_red_dim <- reducedDim(pop_sce, "pca.corrected")
  sce.graph <- buildKNNGraph(t(X_red_dim), k=k)
  louvain.clust <- cluster_louvain(sce.graph)
  louvain.clust.ids <- membership(louvain.clust)
  pop_sce$subcluster <- paste(pop, as.character(louvain.clust.ids), sep="_")
  subclusters <- setNames(pop_sce$subcluster, colnames(pop_sce))
  colData(embryo_sce)["subcluster"][names(subclusters),] <- subclusters
  }

# pop_sce <- runUMAP(pop_sce, dimred="pca.corrected")
subclusters_umap <- data.frame(reducedDim(embryo_sce, "UMAP")) %>%
  mutate(subcluster=embryo_sce$subcluster) %>%
  mutate(subcluster=factor(subcluster, levels=sample(unique(subcluster)))) %>%
  ggplot(aes(X1, X2, color=subcluster)) +
  geom_point(size=0.1) +
  guides(color="none") +
  xlab("UMAP1") + ylab("UMAP2") +
  theme_classic(base_size=18)

Make simulated labels

embryo_sce <- add_synthetic_labels_pop(embryo_sce, pop="Somitic mesoderm", pop_column = "celltype", seed=42, pop_enr=0.8)
true_labels <- ifelse(embryo_sce$Condition2_prob < 0.4, "NegLFC", ifelse(embryo_sce$Condition2_prob > 0.6, "PosLFC", "NotDA"))
colData(embryo_sce)[["true_labels"]] <- true_labels

Testing for DA with GLM on subclusters

condition_col = "synth_labels"
sample_col = "synth_samples"
sce <- embryo_sce

## Make design matrix
design_df <- as.tibble(colData(sce)[c(sample_col, condition_col)]) %>%
    distinct() %>%
    rename(sample=sample_col)
design <- formula(paste('Freq ~', condition_col, "+ offset(log(N_s))", collapse = ' '))  

## Test DA in subclusters
condition_vec <- colData(embryo_sce)[[condition_col]]
sample_labels <- colData(embryo_sce)[[sample_col]]
clust.df <- data.frame("cell_id"=colnames(embryo_sce), "Louvain.Clust"=embryo_sce$subcluster)
clust.df$Sample <- sample_labels
clust.df$Condition <- condition_vec

louvain.count <- table(clust.df$Louvain.Clust, clust.df$Sample)
attributes(louvain.count)$class <- "matrix"
  
df <- melt(louvain.count, varnames=c("cluster", "sample"),  value.name="Freq") %>%
    mutate(cluster=factor(cluster)) %>%
    left_join(design_df, by="sample") %>%
    group_by(sample) %>%
    mutate(N_s=sum(Freq)) %>%
    ungroup() %>%
    group_by(cluster) %>%
    do(model=glm(design, data=., family="poisson")) 
  
res_df <- t(sapply(df$model, function(x) summary(x)$coefficients[nrow(summary(x)$coefficients),]))
colnames(res_df) <- c("logFC","Std. Error", "z value",    "Pval" )
louvain.res <- cbind(df, res_df) %>%
  mutate(FDR=p.adjust(Pval, method = "BH"))
rownames(louvain.res) <- louvain.res$cluster
  
clust.df$logFC <- louvain.res[clust.df$Louvain.Clust, 'logFC']
clust.df$FDR <- louvain.res[clust.df$Louvain.Clust, 'FDR']
embryo_sce$predDA_subclusters <- louvain2output(clust.df, out_type = "labels")

clust.df %>%
  ggplot(aes(logFC, -log10(FDR))) + geom_point()

## Test DA in cell type clusters
clust.df <- data.frame("cell_id"=colnames(embryo_sce), "Louvain.Clust"=embryo_sce$celltype)
clust.df$Sample <- sample_labels
clust.df$Condition <- condition_vec

louvain.count <- table(clust.df$Louvain.Clust, clust.df$Sample)
attributes(louvain.count)$class <- "matrix"
  
df <- melt(louvain.count, varnames=c("cluster", "sample"),  value.name="Freq") %>%
    mutate(cluster=factor(cluster)) %>%
    left_join(design_df, by="sample") %>%
    group_by(sample) %>%
    mutate(N_s=sum(Freq)) %>%
    ungroup() %>%
    group_by(cluster) %>%
    do(model=glm(design, data=., family="poisson")) 
  
res_df <- t(sapply(df$model, function(x) summary(x)$coefficients[nrow(summary(x)$coefficients),]))
colnames(res_df) <- c("logFC","Std. Error", "z value",    "Pval" )
louvain.res <- cbind(df, res_df) %>%
  mutate(FDR=p.adjust(Pval, method = "BH"))
rownames(louvain.res) <- louvain.res$cluster
  
clust.df$logFC <- louvain.res[clust.df$Louvain.Clust, 'logFC']
clust.df$FDR <- louvain.res[clust.df$Louvain.Clust, 'FDR']
embryo_sce$predDA_celltype <- louvain2output(clust.df, out_type = "labels")

clust.df %>%
  ggplot(aes(logFC, -log10(FDR))) + geom_point()

milo_res <- run_milo(embryo_sce, condition_col="synth_labels", sample_col="synth_samples", reduced.dim = "pca.corrected",
        k=50, d = 30)


embryo_sce$predDA_milo <- milo2output(milo_res$Milo, milo_res$DAres, out_type = "labels", alpha = 0.1)

# embryo_sce$predDA <- louvain2output(clust.df, out_type = "labels")

pl_df <- data.frame(reducedDim(embryo_sce, "UMAP")) %>%
  mutate(subcluster=embryo_sce$subcluster) %>%
  mutate(subcluster=factor(subcluster, levels=sample(unique(subcluster)))) %>%
  mutate(predDA_subclusters=embryo_sce$predDA_subclusters,
         predDA_celltype=embryo_sce$predDA_celltype,
         predDA_milo=embryo_sce$predDA_milo,
         trueDA = embryo_sce$true_labels) 

true_pl <- pl_df %>%
  ggplot(aes(X1, X2, color=trueDA)) +
  geom_point(size=0.1) +
  xlab("UMAP1") + ylab("UMAP2") +
  theme_classic(base_size=18) +
  scale_color_manual(values=c(NotDA="grey", PosLFC="Red", NegLFC="Blue"),name="") +
  guides(color="none") +
  ggtitle("True DA")

subcl_plot <- pl_df %>%
  ggplot(aes(X1, X2, color=predDA_subclusters)) +
  geom_point(size=0.1) +
  xlab("UMAP1") + ylab("UMAP2") +
  theme_classic(base_size=18) +
  scale_color_manual(values=c(NotDA="grey", PosLFC="Red", NegLFC="Blue"),name="") +
  guides(color=guide_legend(override.aes = c(size=2))) +
  ggtitle("GLM on subclusters")

cl_plot <- pl_df %>%
  ggplot(aes(X1, X2, color=predDA_celltype)) +
  geom_point(size=0.1) +
  xlab("UMAP1") + ylab("UMAP2") +
  theme_classic(base_size=18) +
  scale_color_manual(values=c(NotDA="grey", PosLFC="Red", NegLFC="Blue"),name="") +
  guides(color="none") +
  ggtitle("GLM on clusters")

milo_pl <- pl_df %>%
  ggplot(aes(X1, X2, color=predDA_milo)) +
  geom_point(size=0.1) +
  xlab("UMAP1") + ylab("UMAP2") +
  theme_classic(base_size=18) +
  scale_color_manual(values=c(NotDA="grey", PosLFC="Red", NegLFC="Blue"), name="") +
  guides(color="none") +
  ggtitle("Milo")

fig2 <- true_pl + milo_pl + cl_plot + subcl_plot +
  plot_layout(ncol=2, nrow=2, guides="collect") 

Compare size of subclusters and size of neighbourhoods

size_fig <- (data.frame(table(embryo_sce$subcluster)) %>%
  ggplot(aes(Freq)) + geom_histogram() +
  theme_classic(base_size = 16) +
   annotate("text", label=paste0("N = ", length(unique(embryo_sce$subcluster))), x=750, y=20, size=5) +
   xlab("Subcluster size") +
  xlim(0, 1000)) /
(plotNhoodSizeHist(milo_res$Milo) +
  xlim(0, 1000) +
   annotate("text", label=paste0("N = ", ncol(nhoods(milo_res$Milo))), x=750, y=600, size=5) +
   xlab("Milo neighbourhood size"))

size_fig

((((subclusters_umap + ggtitle("Subclusters")) /  size_fig) + plot_layout(ncol=1)) |
   fig2 ) +
  plot_layout(ncol=2, widths=c(1,2)) +
  plot_annotation(tag_levels = 'A') +
  ggsave("~/mount/gdrive/milo/Figures/RebuttalFig1_microclusters.png", width = 14, height = 8)